import numpy as np
from mpl_toolkits import mplot3d
import matplotlib.pyplot as plt
import pandas as pd
from python_ai.common.xcommon import sep

pd.set_option('display.max_rows', None, 'display.max_columns', None, 'display.max_colwidth', 1000, 'display.expand_frame_repr', False)

df = pd.read_csv(r'../../../../large_data/ML2/titanic.csv')
print(df[:5])
print(df.info())
m, n = df.shape

sep('na isnull')
isn1 = df.isnull()
isn2 = df.isna()
print(isn1[:5])
print(isn1.equals(isn2))

sep('na notnull')
isnn1 = df.notnull()
isnn2 = df.notna()
print(isnn1[:5])
print(isnn1.equals(isnn2))

# 缺失比例
sep('na rate')
rate = df.isnull().sum(axis=0) / m
print(rate)

# select na
sep('na columns')
ix = (df.isna().sum(axis=0)>0)
col = df.columns[ix]
print(col)  # ATTENTION Series
print(type(col.values))  # ATTENTION ndarray

# fillna
sep('fillna by sklearn')
from sklearn.impute import SimpleImputer
# strategy: mean, median, most_frequent
sim = SimpleImputer(strategy='mean')
df['age'] = sim.fit_transform(df['age'].values.reshape(-1, 1))
ix = (df.isna().sum(axis=0)>0)
col = df.columns[ix]
print(col)  # ATTENTION Series

sep('fillna by pandas')
df['embarked'].fillna(3, inplace=True)
ix = (df.isna().sum(axis=0)>0)
col = df.columns[ix]
print(col)  # ATTENTION Series
